




//ó 1.
//accept 帶  .
//accept loop . .
//ó 1 ReadMe.txt Ͽ ִ.

//ó 2.
//accept queue  .
//   .  ó ϴ.

// ϴ GetQueue ȣϴ   Ѵ.
//WSARecv WSASend GetQueue ȣ   . װ 츮 ϴ thread   .
//Ե. ش 忡 GetQueue ȣ Ƿ. PostQueue Overlapped. Event ٸ Լ ؾ Ұ .

//  1.
//td_AcceptQueue .
//critical section g_socketlist[MAX_SOCKET] Ѵ.

// ó  -   2.
//td_AcceptQueue .
//g_socketlist[MAX_THREAD_ID][MAX_SOCKET] Ѵ.
//Ǳ ߴµ.       ߴ.


// 
//td queue WinProc ϴ  ִ.







// chat_server.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include "chat_server.h"



#include <windows.h>
#include <math.h>

#include <process.h>
#include <conio.h>



#ifdef _DEBUG
#define new DEBUG_NEW
#undef THIS_FILE
static char THIS_FILE[] = __FILE__;
#endif




typedef struct DF_ST
{
	int					IO_TYPE;
	SOCKET				sock;
	//	char val[100];
	PER_SOCKET_CONTEXT*	pSocketContext;
}ST;

int use_accept = 0;
vector<ST> v_accept;

int use_close = 0;
vector<ST> v_close;

int use_sendall = 0;
vector<ST> v_sendall;


SYSTEM_INFO         systemInfo;					//CPU  ˱ ؼ....IOCP    cpu*2 


//  Լ    
//fn_WSAGetLastError("recv Error\n", recv_len, __FUNCTION__, __LINE__);
void fn_WSAGetLastError(char* input_msg, int result, char* function_name, int line)
{
    LPVOID lpMsgBuf;
	DWORD dwWSAErrorCode;
	
	//
	dwWSAErrorCode = WSAGetLastError();
    FormatMessage(  FORMAT_MESSAGE_ALLOCATE_BUFFER|
		FORMAT_MESSAGE_FROM_SYSTEM,
		NULL, dwWSAErrorCode,
		MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
		(LPTSTR)&lpMsgBuf, 0, NULL);
	//    MessageBox(NULL, (LPCTSTR)lpMsgBuf, msg_input, MB_ICONERROR);
	printf("----------------------------------------\n");
	printf("function name     [%s()]\n",	function_name);
	printf("line number       [%d]\n",		line);
    printf("input_msg         [%s]\n",		input_msg);
    printf("result            [%d]\n",		result);
    printf("wsa_error_code    [%d]\n",		dwWSAErrorCode);
    printf("wsa_error_string  %s",			(LPCTSTR)lpMsgBuf);
	printf("----------------------------------------\n");
    LocalFree(lpMsgBuf);
	
	//  ' ǵ Ǿ . Ȳ ° ؼ ϼ. 
	//    system("PAUSE\n");
	//    exit(-1);   //or exit(0);
}








int WINAPI WinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, PSTR szCmdLine, int iCmdShow);
LRESULT CALLBACK WndProc (HWND, UINT, WPARAM, LPARAM) ;
LRESULT CALLBACK WndProc1 (HWND, UINT, WPARAM, LPARAM) ;

static WNDPROC ChildProc = WndProc1;
HINSTANCE      hInstance ;
static HWND    hwndChild ;
static TCHAR * szChildClass = TEXT ("Child1");



LRESULT APIENTRY WndProc (HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
	HANDLE handleConsole = NULL;
	HWND hwndChild;
	WNDCLASS     wndclass ;
	
	switch (message)
	{
	case WM_CREATE:
		//ν 
		//handleConsole = ::GetStdHandle( STD_OUTPUT_HANDLE );
		//		handleConsole = ::GetModuleHandle( NULL );
		//hwnd = (struct HWND__ *)handleConsole;
		
		hInstance = (HINSTANCE) GetWindowLong (hwnd, GWL_HINSTANCE) ;
		
		wndclass.style         = CS_HREDRAW | CS_VREDRAW ;
		wndclass.cbClsExtra    = 0 ;
		wndclass.cbWndExtra    = 0 ;
		wndclass.hInstance     = hInstance ;
		wndclass.hIcon         = NULL ;
		wndclass.hCursor       = LoadCursor (NULL, IDC_ARROW) ;
		wndclass.hbrBackground = (HBRUSH) GetStockObject (WHITE_BRUSH) ;
		wndclass.lpszMenuName  = NULL ;

		{
			wndclass.lpfnWndProc   = ChildProc ;
			wndclass.lpszClassName = szChildClass ;
			
			RegisterClass (&wndclass) ;
			
			hwndChild = CreateWindow (szChildClass, NULL,
										WS_CHILDWINDOW | WS_BORDER | WS_VISIBLE,
										0, 0, 0, 0, 
										hwnd, (HMENU) 0, hInstance, NULL) ;
			ShowWindow(hwndChild, SW_SHOW);
		}

		return 0 ;
	case WM_DESTROY:
		PostQuitMessage (0) ;
		return 0 ;
	}
	return DefWindowProc (hwnd, message, wParam, lParam) ;
}


LRESULT APIENTRY WndProc1 (HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
	HDC        hdc ;
	LPCTSTR str;
	int		size;
	HANDLE handleConsole = NULL;

	switch (message)
	{
	case WM_CREATE:
		handleConsole = ::GetStdHandle( STD_OUTPUT_HANDLE );	// ܼ   ڵ
		//handleConsole = ::GetModuleHandle( NULL );			//  ڵ
		str = "ν \n";
//		_tcsclen(str);											//ڵ TCHAR
		::WriteFile( handleConsole, str, strlen( str ), (unsigned long *)&size, 0 );
//		if( ::AllocConsole() == FALSE )	//ܼ 
//		{
//			::FreeConsole();
//		}
//		::CloseHandle( handleConsole );
//		handleConsole = INVALID_HANDLE_VALUE;
		return 0 ;
	case WM_TIMER:
		hdc = GetDC (hwnd) ;
		ReleaseDC (hwnd, hdc) ;
		return 0 ;
	case WM_DESTROY:
		PostQuitMessage (0) ;
		return 0 ;
	}
	return DefWindowProc (hwnd, message, wParam, lParam) ;
}






/////////////////////////////////////////////////////////////////////////////
// The one and only application object

CWinApp theApp;




SOCKET	sdAccept = INVALID_SOCKET;


UINT td_pump(LPVOID pParam);
UINT td_sendall(LPVOID pParam);

UINT td_window(LPVOID pParam)
{

	//
//	HANDLE handleConsole = ::GetStdHandle( STD_OUTPUT_HANDLE );
	HANDLE handleConsole = ::GetModuleHandle( NULL );
	HWND hwnd = (struct HWND__ *)handleConsole;
	HWND hwndChild = NULL;
	WNDCLASS     wndclass ;
	
	
	MSG          msg ;
	static TCHAR szAppName[] = TEXT ("Multi1") ;
	
	wndclass.style         = CS_HREDRAW | CS_VREDRAW ;
	wndclass.lpfnWndProc   = WndProc ;
	wndclass.cbClsExtra    = 0 ;
	wndclass.cbWndExtra    = 0 ;
	wndclass.hInstance     = hInstance ;
	wndclass.hIcon         = LoadIcon (NULL, IDI_APPLICATION) ;
	wndclass.hCursor       = LoadCursor (NULL, IDC_ARROW) ;
	wndclass.hbrBackground = (HBRUSH) GetStockObject (WHITE_BRUSH) ;
	wndclass.lpszMenuName  = NULL ;
	wndclass.lpszClassName = szAppName ;
	
	if (!RegisterClass (&wndclass))
	{
		MessageBox (NULL, TEXT ("This program requires Windows NT!"),
			szAppName, MB_ICONERROR) ;
		return 0 ;
	}
	
	hwnd = CreateWindow (szAppName, TEXT ("Multitasking Demo"),
							WS_OVERLAPPEDWINDOW,
							CW_USEDEFAULT, CW_USEDEFAULT,
							CW_USEDEFAULT, CW_USEDEFAULT,
							NULL, NULL, hInstance, NULL) ;
	
	ShowWindow(hwnd, SW_SHOW);
	UpdateWindow (hwnd) ;
	
	while (GetMessage (&msg, NULL, 0, 0))
	{
		TranslateMessage (&msg) ;
		DispatchMessage (&msg) ;
	}
	return msg.wParam ;
	
	
}




PER_SOCKET_CONTEXT * p[MAX_WORKER_THREAD][MAX];


int _tmain(int argc, TCHAR* argv[], TCHAR* envp[])
{
	int nRetCode = 0;
	
	// initialize MFC and print and error on failure
	if (!AfxWinInit(::GetModuleHandle(NULL), NULL, ::GetCommandLine(), 0))
	{
		// TODO: change error code to suit your needs
		cerr << _T("Fatal Error: MFC initialization failed") << endl;
		return nRetCode = 1;
	}

	
	
	//
//	_CrtSetDbgFlag(_CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
	
	
	
	
	
    WSADATA             wsaData;					//WSADATA
    PPER_SOCKET_CONTEXT lpPerSocketContext = NULL;	//
    DWORD               dwRecvNumBytes = 0;			//
    DWORD               dwFlags = 0;				//
    int                 nRet;						//üũ  
	
	g_bEndServer = FALSE;
	
	
	
	// WS2_32.DLL    ֵ ʱȭ
	if((nRet = WSAStartup(MAKEWORD(2,2), &wsaData)) !=0)
	{
		printf("WSAStarup failed : %d, %d\n", nRet,GetLastError());
		return 1;
	}//if((nRet = WSAStartup(MAKEWORD(2,2), &wsaData)) !=0)
	
	
	//ȭ  ؼ ũƽŬ Ѵ.
	//ɼ  ũƼŬ  //event, semaphore, mutex... Ŀΰü̹Ƿ...
	InitializeCriticalSection(&g_cs1);
	InitializeCriticalSection(&g_cs_accept);
	InitializeCriticalSection(&g_cs_close);
	InitializeCriticalSection(&g_cs_sendall);	
	InitializeCriticalSection(&g_csRTL);



	GetSystemInfo(&systemInfo);
	
	//completion port Ʈ   completion port Ư ڵ .
	// Լ  : HANDLE CreateIoCompletionPort(HANDLE FileHandle,HANDLE ExistingCompletionPort,DWORD CompletionKey,DWORD NumberOfConcurrentThreads);
	// completion port Ҵ    Ǵµ.. ڿ 0  ϸ CPU    .
	g_hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,0,systemInfo.dwNumberOfProcessors);
	if(NULL == g_hIOCP)
	{
		printf("CreateIoCompletionPort Failed : %d\n", GetLastError());
		return 1;
	}
	
	g_hIOCP2 = CreateIoCompletionPort(INVALID_HANDLE_VALUE,NULL,0,systemInfo.dwNumberOfProcessors);
	if(NULL == g_hIOCP2)
	{
		printf("CreateIoCompletionPort Failed : %d\n", GetLastError());
		return 1;
	}



	//IOCP   CPU *2 Ѵ... ص ȴٰ ...õ 鰡 پ...*2ϸ ȴ...
	//CPU *2̻ ڸ Ѿ Context switch ϰ ɸ. //ҿ ϴ ̴...
	g_dwThreadCount = systemInfo.dwNumberOfProcessors*2;
	//IOCPť   о ̴  ϴ .
	for(DWORD dwCPU = 0; dwCPU < g_dwThreadCount; dwCPU++)
	{
		//CPU ι踸ŭ  
		DWORD dwThreadID;
		g_hThreads[dwCPU] = CreateThread(NULL, 0, EchoThread, g_hIOCP, 0, &dwThreadID);
		if(g_hThreads[dwCPU] == NULL)
		{
			printf("CreateThread failed : %d, %d°   . \n", GetLastError(), dwCPU);
			CleanUp();
			return 1;
		}
		g_hThreads[dwCPU] = CreateThread(NULL, 0, AcceptThread, g_hIOCP2, 0, &dwThreadID);
		if(g_hThreads[dwCPU] == NULL)
		{
			printf("CreateThread failed : %d, %d°   . \n", GetLastError(), dwCPU);
			CleanUp();
			return 1;
		}
	}//for(DWORD dwCPU = 0; dwCPU < g_dwThreadCount; dwCPU++)

	
	
	// Ѵ.
	if(!CreateListenSocket())
	{
		printf("CreateListenSocket() \n");
		CleanUp();
		return 1;
	}
	
	

	//
	int j=0;
	int i=0;
	for(j=0; j<MAX_WORKER_THREAD; j++)
	{
		for(i=0; i<MAX; i++)
		{
			p[j][i] = NULL;
		}
	}
	
	
	//   
	//޸𸮸 ̸  д.
	for(j=0; j<MAX_WORKER_THREAD; j++)
	{
		for(i=0; i<MAX; i++)
		{
			p[j][i] = CtxtAllocate(0, ClientIoRead);
			p[j][i]->Socket = 0;
			p[j][i]->thread_id = 0;
		}
	}


	
//	_beginthread( (void (__cdecl *)(void *))td_window, 0, (void *)0);
	
//	_beginthread( (void (__cdecl *)(void *))td_sendall, 0, (void *)0);
//	_beginthread( (void (__cdecl *)(void *))td_pump, 0, (void *)0);
	_beginthreadex( NULL, 0, (unsigned int (__stdcall *)(void *))td_sendall, 0, (unsigned int)0, 0);

	
	
	
	
	//TRUE϶ ؼ accept ٸ.
	while(g_bEndServer == FALSE)
	{
		fd_set fds;
		struct timeval tv = { 0, 1 };		//100  0.1 
		
		FD_ZERO( &fds );
		FD_SET( g_sdListen, &fds );
		
		::select( 0, &fds, 0, 0, &tv );
		if( FD_ISSET( g_sdListen, &fds ) )
		{
			struct sockaddr_in fromAddr;
			int size = sizeof( fromAddr );
			
			//sdAccept = ::accept( g_sdListen, ( struct sockaddr* )&fromAddr, &size );
			//			printf( "Accepted a client : %s\n", ::inet_ntoa( fromAddr.sin_addr ) );
			
			sdAccept = WSAAccept(g_sdListen, NULL, NULL, NULL, WSA_FLAG_OVERLAPPED);
			//sdAccept = accept(g_sdListen, NULL, NULL);
			if(sdAccept == SOCKET_ERROR)
			{
				//1455				ERROR_COMMITMENT_LIMIT
				//					WSAENOTSOCK		10038L
				// 6000		WSAENOBUFS		10055L
				fn_WSAGetLastError("WSASend: \n", nRet, __FILE__, __LINE__);
				printf("WSAAccept : %d Error \n", WSAGetLastError());
				//				CleanUp();
				//				return 1; 
				continue;
			}
		}
		else
		{
			continue;
		}
		
		
		int len = 0;
		
		
		
		

		//Accept IOCP
//		fn_enter_cs();
		ST * pst = (ST*) malloc (sizeof(ST));
		pst->sock = sdAccept;
		pst->pSocketContext = lpPerSocketContext;

		BOOL ret = PostQueuedCompletionStatus(g_hIOCP2, 0, 1, (struct _OVERLAPPED *) pst);
//		fn_leave_cs();

		
		
		
		
		
	}//while(g_bEndServer == FALSE)
	
	printf("while(g_bEndServer == FALSE) \n");
	CleanUp();
	
	return nRetCode = 1;
}




UINT td_pump(LPVOID pParam)
{

	return 0;
}





DWORD dwCheck[MAX_WORKER_THREAD];


UINT td_sendall(LPVOID pParam)
{
	while(1)
	{
		try
		{
			//
			getch();
			printf("getch()\n");

			int i=0;
			for(i=0; i<MAX_WORKER_THREAD; i++)
			{
				dwCheck[i] = 0;
			}


			WSAEVENT EventArray[1];
			for(i=0; i<MAX_WORKER_THREAD; i++)
			{
				//
				WSAOVERLAPPED	ov;
				ov.Internal = STATUS_PENDING;

				//
				WSAEVENT AcceptEvent = WSACreateEvent();
				ov.hEvent = AcceptEvent;

				WSASetEvent(AcceptEvent);
				WSAResetEvent(AcceptEvent);

				BOOL ret = PostQueuedCompletionStatus(g_hIOCP2, 0, 3, &ov);

				WSACloseEvent(AcceptEvent);
			}
		}
		catch(...)
		{
			printf("td exception\n");
			OutputDebugString("td exception\n");
		}
	}
	return 0;
}











//	{
//		// 3 ü 10 
//		ST_A * pa[3][10];
//		int i=0;
//		int j=0;
//		for(j=0; j<3; j++)
//		{
//			for(i=0; i<10; i++)
//			{
//				pa[j][i] = (ST_A*) malloc (sizeof(ST_A));
//				pa[j][i]->a = i;
//				printf("[%d %d :  %d]\n", j, i, pa[j][i]->a);
//				free(pa[j][i]);
//			}
//			printf("\n");
//		}
//	}




DWORD dwTD_ID[MAX_WORKER_THREAD];
int td_cnt = 0;

DWORD WINAPI AcceptThread(LPVOID WorkThreadContext)
{

	HANDLE hIOCP = (HANDLE)WorkThreadContext;
	BOOL bSuccess = FALSE; //IOCP ť 񵿱 I/O ϳ о   
	int nRet;
	LPOVERLAPPED lpOverlapped = NULL;
	PPER_SOCKET_CONTEXT lpPerSocketContext = NULL;
	PPER_SOCKET_CONTEXT temp = NULL; //  Ŭ̾Ʈ   ؼ 
	PPER_IO_CONTEXT lpIOContext = NULL;
	WSABUF buffRecv;
	WSABUF buffSend;
	DWORD	dwRecvNumBytes = 0;
	DWORD   dwSendNumBytes = 0;
	DWORD	dwFlags		   = 0;
	DWORD	dwIoSize;
	POSITION pos;

	int i=0;
	int read = 0;
	int loop_cnt = 0;




	// ID ϴ 
	EnterCriticalSection(&g_csRTL);		//ӿ EnterCriticalSection .
	if(dwTD_ID[td_cnt] == 0)
	{
	}
	else
	{
		td_cnt++;
	}
	int		id	= td_cnt;
	dwTD_ID[id] = GetCurrentThreadId();
	LeaveCriticalSection(&g_csRTL);

	printf("[id : %d      td : %d]\n", id, dwTD_ID[id]);




	//
	for(i=0; i<MAX_WORKER_THREAD; i++)
	{
		dwCheck[i] = 0;
	}




	while(TRUE)
	{
		
		try
		{

			bSuccess = GetQueuedCompletionStatus(hIOCP, &dwIoSize, (LPDWORD)&lpPerSocketContext, &lpOverlapped, INFINITE);
			if(!bSuccess)
			{
				//printf("GetQueueCompletionStatus : %d Error \n",GetLastError());
			}//if(!bSuccess)


			//    .
			if(lpPerSocketContext == NULL)
			{
				return 0;
			}//	if(lpPerSocketContext == NULL)
			
			if(g_bEndServer)
			{
				return 0;
			}//if(g_bEndServer)




			//--------------------------------------------------------------
			//ACCEPT
			//--------------------------------------------------------------
			if((int)lpPerSocketContext == 1)
			{
//				SOCKET sock = (SOCKET) lpOverlapped;
				ST * pst = (ST*) lpOverlapped;
				SOCKET sock = pst->sock;

				free( pst );		//delete  .
				pst = NULL;



				//
//				printf("[ id : %d     : %d      td : %d]\n", id, sock, dwTD_ID[id]);

				//
				PPER_SOCKET_CONTEXT lpPerSocketContext = NULL;

				//
//				lpPerSocketContext = CtxtAllocate(sock, ClientIoRead);
//				if(0 <= sock && sock <= 655355)


//	EnterCriticalSection(&g_csRTL);
				//. ִ´.
				for(i=0; i<MAX; i++)
				{
					if(p[id][i] != NULL)
					{
						if(p[id][i]->Socket == 0)
						{
							p[id][i]->thread_id	= id;
							p[id][i]->Socket	= 0;
							lpPerSocketContext	= p[id][i];
							break;
						}
					}
				}
//	LeaveCriticalSection(&g_csRTL);

				//
//				printf("[ id : %d          : %d    ]\n", id, sock);


				// ϳ  ̸. ٸ  Ѱܾ Ѵ.
				if(lpPerSocketContext == NULL)
				{
					printf("FULL [%d]\n", id);
					closesocket(sock);
					continue;
				}

				// Ҵ ü  g_hIOCP Ѵ.
				g_hIOCP = CreateIoCompletionPort((HANDLE)sock, g_hIOCP, (DWORD)lpPerSocketContext, systemInfo.dwNumberOfProcessors);
				if(g_hIOCP == NULL)
				{
					printf("CreateIoCompletionPort() : %d Error\n", GetLastError());

					closesocket(sock);

					if(lpPerSocketContext != NULL)
					{
						lpPerSocketContext->Socket		= 0;
						lpPerSocketContext->thread_id	= id;
					}

//					free(lpPerSocketContext->pIOContext);
//					lpPerSocketContext->pIOContext = NULL;
//					free(lpPerSocketContext);
//					lpPerSocketContext = NULL;
					continue;
				}

				// Ʈ completion port  context ٿ .

				if(lpPerSocketContext == NULL)
				{
					closesocket(sock);
					continue;
				}


				//
				lpPerSocketContext->thread_id	= id;
				lpPerSocketContext->Socket		= sock;
//				p[id][i]						= lpPerSocketContext;




				//
//				EnterCriticalSection(&g_csRTL);
//				for(i=0; i<MAX; i++)
//				{
//					if(p[id][i] == NULL)
//					{
//						p[id][i] = lpPerSocketContext;
//						break;
//					}
//				}
//				LeaveCriticalSection(&g_csRTL);


		
				
				//WSABUF ʱȭ ϱ
				int len = lpPerSocketContext->pIOContext->wsabuf.len;
				memset((char*)&(lpPerSocketContext->pIOContext->Buffer), 0x00, len);
				
				nRet = WSARecv(sock, &(lpPerSocketContext->pIOContext->wsabuf), 1, &dwRecvNumBytes, &dwFlags,
					&(lpPerSocketContext->pIOContext->Overlapped), NULL);
				
				if(nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError()))
				{
					//
					for(i=0; i<MAX; i++)
					{
						int tid = lpPerSocketContext->thread_id;
						tid =id;
						if(p[tid][i] != NULL)
						{
							if(p[tid][i]->Socket == sock)
							{
									//
									closesocket(sock);
									p[tid][i]->Socket = 0;
									p[tid][i]->thread_id = 0;
									break;
							}//if(p[id][i]->Socket == sock)
						}//if(p[id][i] != NULL)
					}//for(i=0; i<MAX; i++)

					//			printf("WSARecv() Failed : %d Error \n", WSAGetLastError());
//					CloseClient(lpPerSocketContext);

				}//	if(nRet == SOCKET_ERROR && (ERROR_IO_PENDING != WSAGetLastError())
				
				
				continue;
			}



			//--------------------------------------------------------------
			//CLOSE
			//--------------------------------------------------------------
			if((int)lpPerSocketContext == 2)
			{
				//
				ST * pst = (ST*) lpOverlapped;
				SOCKET sock = pst->sock;
				temp = pst->pSocketContext;

				free( pst );		//delete  .
				pst = NULL;


				//
				PPER_SOCKET_CONTEXT lpPerSocketContext = NULL;
//				temp = (PER_SOCKET_CONTEXT*) lpOverlapped;
//				SOCKET sock = (SOCKET) lpOverlapped;


//	EnterCriticalSection(&g_csRTL);
				for(i=0; i<MAX; i++)
				{
					if(p[id][i] != NULL)
					{
						if(p[id][i]->Socket == sock)
						{
								//
//								closesocket(sock);

								//
								p[id][i]->Socket = 0;

//								//
//								if(p[id][i]->pIOContext)
//								{
//									free(p[id][i]->pIOContext);
//									p[id][i]->pIOContext = NULL;
//								}
//								free(p[id][i]);
//								p[id][i] = NULL;

								//
//								fn_enter_cs();
//								fn_leave_cs();
								break;
						}//if(p[id][i]->Socket == sock)

					}//if(p[id][i] != NULL)
				}//for(i=0; i<MAX; i++)

//	LeaveCriticalSection(&g_csRTL);
				continue;
			}
			
			
			
			
			//--------------------------------------------------------------
			//SEND ALL
			//--------------------------------------------------------------
			if((int)lpPerSocketContext == 3)
			{
				
				
				//
				//	EnterCriticalSection(&g_csRTL);
				//			dwCheck[id] = 1;
				//
				//			int cnt = 0;
				//			for(i=0; i<16; i++)
				//			{
				//				if(dwCheck[i] == 1)
				//				{
				//					cnt++;
				//				}
				//			}
				//			printf("[GetQueueCompletionStatus()  id : %d     td : %d      cnt : %d]\n", id, dwTD_ID[id], cnt);
				//
				////			for(i=0; i<id; i++)
				////			{
				////				printf(".");
				////			}
				////			printf("%d\n", id);
				//	LeaveCriticalSection(&g_csRTL);
				//
				//
				
				
				
				
				
				
				//
				int i=0;
				int sendcount_size = 1000;
				
				//
				if(loop_cnt >= MAX)
				{
					loop_cnt = 0;
				}
				
				
				
				
				//   ϱ
				int max_cnt = 0;
				int j=0;
//				for(j=0; j<MAX_WORKER_THREAD; j++)
				j = id;
				{
					for(i=0; i<MAX; i++)
					{
						if(p[j][i] != NULL)
						{
							if(p[j][i]->Socket != 0)
							{
								max_cnt++;
							}
						}
					}
				}
				
				
				
				//
				int exit = 0;
//				for(j=0; j<MAX_WORKER_THREAD; j++)
				{
					j = id;

					for(i=0; i<MAX; i++)
						//for(i=loop_cnt; i<loop_cnt+sendcount_size; i++)
					{
						if(p[j][i] != NULL)
						{
							if(p[j][i]->Socket != 0)
							{
								//printf("sendall sock p[id][i] : %d\n", p[id][i]->Socket);
							}
						}
						
						if(p[j][i] != NULL)
						{
							if(p[j][i]->Socket != 0 && p[j][i]->Socket > 0)
								//if(p[j][i]->Socket != 0)
							{
								ST * pst			= (ST*) malloc (sizeof(ST));
								pst->sock			= p[j][i]->Socket;
								pst->pSocketContext	= p[j][i];
//	printf("sendall sock p[id][i] : %d\n", p[id][i]->Socket);

								BOOL ret = PostQueuedCompletionStatus(g_hIOCP, 0, 1, (struct _OVERLAPPED *) pst);

								
//
//								//
//								temp = (PER_SOCKET_CONTEXT*) p[j][i];		//pst->pSocketContext;
//								
//								//WSASend
//								if(temp != NULL)
//								{
//									if(temp->Socket != 0)
//									{
//										//						memcpy(temp->pIOContext->Buffer, lpIOContext->Buffer, dwIoSize);
//										strcpy(temp->pIOContext->Buffer, "all_send");
//										dwIoSize = strlen("all_send");
//										
//										temp->pIOContext->IOOperation = ClientIoWrite;
//										temp->pIOContext->nTotalBytes = dwIoSize;
//										temp->pIOContext->nSentBytes = 0;
//										temp->pIOContext->wsabuf.len = dwIoSize;
//										
//										nRet = WSASend(temp->Socket, &temp->pIOContext->wsabuf, 1, &dwSendNumBytes, dwFlags, &(temp->pIOContext->Overlapped), NULL); 
//										
//										if (SOCKET_ERROR == nRet)
//										{
//											//							fn_WSAGetLastError("WSASend: \n", nRet, __FILE__, __LINE__);
//											DWORD dwError = WSAGetLastError();
//											//							printf("WSASend - dwError [%d]\n", dwError);
//											if(dwError == ERROR_IO_PENDING)			//997L
//											{
//											}
//											else if(dwError == WSAEWOULDBLOCK)		//10035L
//											{
//											}
//											else if(dwError == WSAENOTSOCK)			//10038L
//											{
//												//
//												for(i=0; i<MAX; i++)
//												{
//													int tid = temp->thread_id;
//													tid =id;
//													if(p[tid][i] != NULL)
//													{
//														if(p[tid][i]->Socket == temp->Socket)
//														{
//															//
////															closesocket(p[tid][i]->Socket);
////															p[tid][i]->Socket = 0;
////															p[tid][i]->thread_id = 0;
//															break;
//														}//if(p[id][i]->Socket == sock)
//													}//if(p[id][i] != NULL)
//												}//for(i=0; i<MAX; i++)
//												
//												//								CloseClient(temp);					//
//											}
//											else if(dwError == WSAECONNRESET)		//10054L
//											{
//												//
//												for(i=0; i<MAX; i++)
//												{
//													int tid = temp->thread_id;
//													tid =id;
//													if(p[tid][i] != NULL)
//													{
//														if(p[tid][i]->Socket == temp->Socket)
//														{
//															//
////															closesocket(p[tid][i]->Socket);
////															p[tid][i]->Socket = 0;
////															p[tid][i]->thread_id = 0;
//															break;
//														}//if(p[id][i]->Socket == sock)
//													}//if(p[id][i] != NULL)
//												}//for(i=0; i<MAX; i++)
//												
//												//								CloseClient(temp);
//											}
//											else
//											{
//												//
//												for(i=0; i<MAX; i++)
//												{
//													int tid = temp->thread_id;
//													tid =id;
//													if(p[tid][i] != NULL)
//													{
//														if(p[tid][i]->Socket == temp->Socket)
//														{
//															//
////															closesocket(p[tid][i]->Socket);
////															p[tid][i]->Socket = 0;
////															p[tid][i]->thread_id = 0;
//															break;
//														}//if(p[id][i]->Socket == sock)
//													}//if(p[id][i] != NULL)
//												}//for(i=0; i<MAX; i++)
//
//												//CloseClient(temp);
//											}
//										}//if(dwError == ERROR_IO_PENDING)			//997L
//										
//									}//if (SOCKET_ERROR == nRet)
//									
//								}//if(temp != NULL)
								
								
								
								
								
								
							}//if(p[j][i]->Socket != 0 && p[j][i]->Socket > 0)
								
						}//if(p[j][i] != NULL)
								
					}//for(i=0; i<MAX; i++)
								
				}//for(j=0; j<MAX_WORKER_THREAD; j++)
				
				
				loop_cnt += sendcount_size;
				
				printf("ü ޽ : %d [%4d][%6d]\n", max_cnt, id, GetCurrentThreadId());
				
				continue;
			}//	if((int)lpPerSocketContext == 3)
			
		}
		catch(...)
		{
			printf("accept exception\n");
			OutputDebugString("accept exception\n");
		}
		
	}//while(TRUE)

	return 0;
}




DWORD WINAPI EchoThread(LPVOID WorkThreadContext)
{
	HANDLE hIOCP = (HANDLE)WorkThreadContext;
	BOOL bSuccess = FALSE; //IOCP ť 񵿱 I/O ϳ о   
	int nRet;
	LPOVERLAPPED lpOverlapped = NULL;
	PPER_SOCKET_CONTEXT lpPerSocketContext = NULL;
	PPER_SOCKET_CONTEXT temp = NULL; //  Ŭ̾Ʈ   ؼ 
	PPER_IO_CONTEXT lpIOContext = NULL;
	WSABUF buffRecv;
	WSABUF buffSend;
	DWORD	dwRecvNumBytes = 0;
	DWORD   dwSendNumBytes = 0;
	DWORD	dwFlags		   = 0;
	DWORD	dwIoSize;
	POSITION pos;

	int i=0;
	int read = 0;
	int loop_cnt = 0;



//
//	// ID ϴ 
//	EnterCriticalSection(&g_csRTL);		//ӿ EnterCriticalSection .
//	if(dwTD_ID[td_cnt] == 0)
//	{
//	}
//	else
//	{
//		td_cnt++;
//	}
//	int		id	= td_cnt;
//	dwTD_ID[id] = GetCurrentThreadId();
//	LeaveCriticalSection(&g_csRTL);
//
//	printf("[id : %d      td : %d]\n", id, dwTD_ID[id]);
//
//
//
//
//	//
//	for(i=0; i<MAX_WORKER_THREAD; i++)
//	{
//		dwCheck[i] = 0;
//	}

	
	
	
	while(TRUE)
	{
		
		try
		{
			bSuccess = GetQueuedCompletionStatus(hIOCP, &dwIoSize, (LPDWORD)&lpPerSocketContext, &lpOverlapped, INFINITE);
			if(!bSuccess)
			{
				//printf("GetQueueCompletionStatus : %d Error \n",GetLastError());
			}//if(!bSuccess)




			//    .
			if(lpPerSocketContext == NULL)
			{
				return 0;
			}//	if(lpPerSocketContext == NULL)
			
			if(g_bEndServer)
			{
				return 0;
			}//if(g_bEndServer)







			//PostQueuedCompletionStatus - ALL SEND
			if((int)lpPerSocketContext == 1)
			{
				ST * pst;
				pst = (ST*) lpOverlapped;
				temp = pst->pSocketContext;

				free( pst );		//delete  .
				pst = NULL;


				//
//				temp = (PER_SOCKET_CONTEXT*) lpOverlapped;

				//WSASend
				if(temp != NULL)
				{
					if(temp->Socket != 0)
					{
						//						memcpy(temp->pIOContext->Buffer, lpIOContext->Buffer, dwIoSize);
						strcpy(temp->pIOContext->Buffer, "all_send");
						dwIoSize = strlen("all_send");
						
						temp->pIOContext->IOOperation = ClientIoWrite;
						temp->pIOContext->nTotalBytes = dwIoSize;
						temp->pIOContext->nSentBytes = 0;
						temp->pIOContext->wsabuf.len = dwIoSize;
						
						nRet = WSASend(temp->Socket, &temp->pIOContext->wsabuf, 1, &dwSendNumBytes, dwFlags, &(temp->pIOContext->Overlapped), NULL); 
						
						if (SOCKET_ERROR == nRet)
						{
							//							fn_WSAGetLastError("WSASend: \n", nRet, __FILE__, __LINE__);
							DWORD dwError = WSAGetLastError();
							//							printf("WSASend - dwError [%d]\n", dwError);
							if(dwError == ERROR_IO_PENDING)			//997L
							{
							}
							else if(dwError == WSAEWOULDBLOCK)		//10035L
							{
							}
							else if(dwError == WSAENOTSOCK)			//10038L
							{
								CloseClient(temp);
							}
							else if(dwError == WSAECONNRESET)		//10054L
							{
								CloseClient(temp);
							}
							else
							{
								CloseClient(temp);
							}
						}//if (SOCKET_ERROR == nRet)
						
					}
					
				}//if(temp != NULL)
				
				continue;
			}
			

			//Ŭ̾Ʈ   .. 
			if(!bSuccess || (bSuccess && (0 == dwIoSize)))
			{
				//			char c[100];
				//			memset(c, 0x00, 100);
				//			sprintf(c, "Ŭ̾Ʈ   : %d \n", lpPerSocketContext->Socket);
				//			printf("%s", c);
				
				//
				CloseClient(lpPerSocketContext);
				continue;
			}//	if(!bSuccess || (bSuccess && (0 == dwIoSize)))]
			
			
			//
			lpIOContext = (PPER_IO_CONTEXT)lpOverlapped;
			
			
			switch(lpIOContext->IOOperation)
			{
			case ClientIoAllSend:
				
				break;
				
			case ClientIoRead: //б ۾ΰ? Ŭ̾ƮԼ Data  ΰ?
				
				//shint
				//			printf("%s ޾Ұ ̸  Ŭ̾Ʈ鿡 մϴ.\n", lpIOContext->wsabuf.buf);
				
				//			memset(lpIOContext->wsabuf.buf, 0x00, sizeof(lpIOContext->wsabuf.buf));
				//			memset(lpIOContext->Buffer, 0x00, sizeof(lpIOContext->Buffer));
				
				temp = lpPerSocketContext;
				if(temp != NULL)
				{
					if(temp->Socket != 0)
					{
						memcpy(temp->pIOContext->Buffer, lpIOContext->Buffer, dwIoSize);
						
						temp->pIOContext->IOOperation = ClientIoWrite;
						temp->pIOContext->nTotalBytes = dwIoSize;
						temp->pIOContext->nSentBytes = 0;
						temp->pIOContext->wsabuf.len = dwIoSize;
						
						nRet = WSASend(temp->Socket, &temp->pIOContext->wsabuf, 1, &dwSendNumBytes, dwFlags, &(temp->pIOContext->Overlapped), NULL); 
						
						if (SOCKET_ERROR == nRet)
						{
							//							fn_WSAGetLastError("WSASend: \n", nRet, __FILE__, __LINE__);
							DWORD dwError = WSAGetLastError();
							//							printf("WSASend - dwError [%d]\n", dwError);
							if(dwError == ERROR_IO_PENDING)			//997L
							{
							}
							else if(dwError == WSAEWOULDBLOCK)		//10035L
							{
							}
							else if(dwError == WSAENOTSOCK)			//10038L
							{
								//
								CloseClient(temp);
							}
							else if(dwError == WSAECONNRESET)		//10054L
							{
								CloseClient(temp);
							}
							else
							{
								CloseClient(temp);
							}
						}//if (SOCKET_ERROR == nRet)
					}
				}//if(temp != NULL)
				break;
				
				
				
			case ClientIoWrite:


				lpIOContext->IOOperation	= ClientIoRead; 
				dwRecvNumBytes				= 0;
				dwFlags					= 0;
				buffRecv.buf				= lpIOContext->Buffer;
				buffRecv.len				= MAX_BUFF_SIZE;

				nRet = WSARecv(lpPerSocketContext->Socket, &buffRecv, 1, &dwRecvNumBytes, &dwFlags, &lpIOContext->Overlapped, NULL);
				if (SOCKET_ERROR == nRet)
				{
					//				fn_WSAGetLastError("WSARecv: \n", nRet, __FILE__, __LINE__);
					DWORD dwError = WSAGetLastError();
					//				printf("WSARecv - dwError [%d]\n", dwError);
					if(dwError == ERROR_IO_PENDING)			//997L
					{
					}
					else if(dwError == WSAEWOULDBLOCK)		//10035L
					{
					}
					else if(dwError == WSAENOTSOCK)			//10038L
					{
						CloseClient(lpPerSocketContext);
					}
					else if(dwError == WSAECONNRESET)		//10054L
					{
						CloseClient(lpPerSocketContext);
					}
					else if(dwError == WSAENOBUFS)			//10055L
					{
					}
					else
					{
						CloseClient(lpPerSocketContext);
					}
				}//if (SOCKET_ERROR == nRet)
				
				break;
				
			}//switch(lpIOContext->IOOperation)
			
			
		}
		catch(...)
		{
			printf("exception\n");
			OutputDebugString("exception\n");
		}
		
	}//while(TRUE)
	
	return 0;
}






void CtxtListDeleteFrom(PPER_SOCKET_CONTEXT lpPerSocketContext)
{
	return;
}



//شϴ Ŭ̾ Ų.
VOID CloseClient(PPER_SOCKET_CONTEXT lpPerSocketContext)
{
	if (lpPerSocketContext)
	{
		
		int nReturn = 0;

		//		// 6. ͺŷ   
		//		unsigned long argp = 1;
		//		nReturn = ioctlsocket(lpPerSocketContext->Socket, FIONBIO, &argp);
		//		if (nReturn == SOCKET_ERROR)
		//		{
		//			//10038
		////			fn_WSAGetLastError("ioctlsocket: \n", nReturn, __FILE__, __LINE__);
		////			printf("ioctlsocket . Error No.%d\n", WSAGetLastError());
		//		}
		
		//		//
		//		::shutdown( lpPerSocketContext->Socket, SD_BOTH );
		
		//		WSAIsBlocking();
		//		//WSACancelAsyncRequest(IN HANDLE hAsyncTaskHandle);
		//		WSACancelBlockingCall();
		//		//WSASetBlockingHook(IN FARPROC lpBlockFunc);
		//		WSAUnhookBlockingHook();
		//
		//		WSARecvDisconnect(lpPerSocketContext->Socket, &lpPerSocketContext->pIOContext->wsabuf);
		//		WSASendDisconnect(lpPerSocketContext->Socket, &lpPerSocketContext->pIOContext->wsabuf);



		//
		SOCKET sock = lpPerSocketContext->Socket;

		//
//		printf("[  : %d      td : %d]\n", sock, GetCurrentThreadId());



		//Accept IOCP
		//fn_enter_cs();
		ST * pst			= (ST*) malloc (sizeof(ST));
		pst->sock			= lpPerSocketContext->Socket;
		pst->pSocketContext	= lpPerSocketContext;

		BOOL ret = PostQueuedCompletionStatus(g_hIOCP2, 0, 2, (struct _OVERLAPPED *) pst);
		//fn_leave_cs();



		//
		closesocket(sock);
		lpPerSocketContext->Socket = 0;		//INVALID_SOCKET;







//		use_close = 1;
//
//		EnterCriticalSection(&g_cs_close);
//		ST st;
//		st.IO_TYPE			= 2;
//		st.sock				= lpPerSocketContext->Socket;
//		st.pSocketContext	= lpPerSocketContext;
//		v_close.push_back(st);
//		LeaveCriticalSection(&g_cs_close);
//
//		free(lpPerSocketContext->pIOContext);
//		lpPerSocketContext->pIOContext = NULL;
//		free(lpPerSocketContext);
//		lpPerSocketContext = NULL;
//		use_close = 0;



		
		//
//		CtxtListDeleteFrom(lpPerSocketContext);

		
		//		printf("\n");
		
	}
    else 
    {
		printf("CloseClient: lpPerSocketContext is NULL\n");
    }
    return;    
}










//Ʈ  Ѵ. //   Ŭ̾Ʈ ݴ´.
void CtxtListFree()
{
	while(!g_CtxtList.IsEmpty())
	{
		PPER_SOCKET_CONTEXT lpPerSocketContext;
		lpPerSocketContext = (PPER_SOCKET_CONTEXT)g_CtxtList.RemoveHead();
		if(lpPerSocketContext->pIOContext)
		{
			free(lpPerSocketContext->pIOContext);
			lpPerSocketContext->pIOContext = NULL;
		}
		free(lpPerSocketContext);
		lpPerSocketContext = NULL;
	}
}





// , Ʈ , IOCP ,    
void CleanUp()
{
	//
	if(g_hIOCP)
	{
		//带  Ѵ.
		for(DWORD i = 0; i < g_dwThreadCount; i++)
		{
			//
			PostQueuedCompletionStatus(g_hIOCP, 0, 0, NULL);
		}//for(DWORD i = 0; i < g_dwThreadCount; i++)
	}//if(g_hIOCP)
	
	// 尡 ߴ ȮѴ.
	if(WAIT_OBJECT_0 != WaitForMultipleObjects(g_dwThreadCount, g_hThreads, TRUE, 1000))
	{
		printf("WaitForMultipleObjects failed: %d\n", GetLastError());
	}//	if(WAIT_OBJECT_0 != WaitForMultipleObjects(g_dwThreadCount, g_hThreads, TRUE, 1000))
	
	else
	{
		for (DWORD i = 0; i < g_dwThreadCount; i++) //带  ݴ´.
		{
			if (g_hThreads[i] != INVALID_HANDLE_VALUE) CloseHandle(g_hThreads[i]);
			{
				g_hThreads[i] = INVALID_HANDLE_VALUE;
			}//	if (g_hThreads[i] != INVALID_HANDLE_VALUE) CloseHandle(g_hThreads[i]);
		}//for (DWORD i = 0; i < g_dwThreadCount; i++)
		
	}//	else
	
	CtxtListFree();// Ŭ̾Ʈ Ѵ.
	
    if (g_hIOCP)    
    {
        CloseHandle(g_hIOCP);//IOCP  ڵ ݴ´. //IOCP Ѵ.
        g_hIOCP = NULL;
    }
	
    if (g_sdListen != INVALID_SOCKET) 
    {
        closesocket(g_sdListen); //  Ѵ.
        g_sdListen = INVALID_SOCKET;
    }
	
    
    
    DeleteCriticalSection(&g_cs1); //ũƽü Ѵ.
	DeleteCriticalSection(&g_cs_accept);
	DeleteCriticalSection(&g_cs_close);
	DeleteCriticalSection(&g_cs_sendall);	
	DeleteCriticalSection(&g_csRTL);
    
    
	WSACleanup();// ̹ Ѵ.
	
}
